
let source_name = "新浪新闻";
let myEncoding = "utf-8";
let seedURL = 'https://www.sina.com.cn/';
let myURL = "";
let seedURL_format = "$('*')";
// 正则表达式在这
let url_reg = /news.sina.com.cn\/.\/(\d{4})-(\d{2})-(\d{2})\/doc-[a-z0-9]{10,}.shtml/;
let url_shipin=/shipin/;


let fs = require('fs');				
let myRequest = require('request');
let myCheerio = require('cheerio');	
let https = require('https');		
let myIconv = require('iconv-lite');
let superagent = require('superagent');	
require('date-utils');				
let mysql = require('./mysql.js');

let schedule=require('node-schedule');
let headers = {
    'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
};


function request(url, callback) {
    let options = {
        url: url,
        encoding: null,
        headers: headers,
   
        timeout: 10000
    }
    myRequest(options, callback);
}

//！定时执行
/*{var rule = new schedule.RecurrenceRule();
var times = [0, 12]; //每天2次自动执行
var times2 = 5; //定义在第几分钟执行
rule.hour = times;
rule.minute = times2;}

//定时执行httpGet()函数
schedule.scheduleJob(rule, function() {
    seedget();
});*/

//function seedget(){
request(seedURL, function(err, res, body) {
    
   
    let html = myIconv.decode(body, myEncoding);
   
    let $ = myCheerio.load(html, {decodeEntities: true});
    let seedurl_news;
    try {
        seedurl_news = eval(seedURL_format);
    } catch(e) {console.log('html模块识别出错' + e)};
    seedurl_news.each(function(i, e) {
        try {
            
            let href = "";
            href = $(e).attr("href");
            if (typeof href == "undefined") {
                return;
            }
            myURL = href;
            
            if (url_shipin.test(myURL)) {return;}
            if (href.toLowerCase().indexOf('https://') >= 0 || href.toLowerCase().indexOf('https://') >= 0) myURL = href;
            else if (href.startsWith('//')) myURL = 'https:' + href;
            else myURL = seedURL.substr(0, seedURL.lastIndexOf('/') + 1) + href;
            
        }  catch (e) {console.log('识别种子页面中的新闻链接出错：' + e)}
        
        if (!url_reg.test(myURL) || url_shipin.test(myURL)) return;
        newsGet(myURL);
        })
    });

//}


function newsGet(myURL) {   
    
    superagent.get(myURL).end((err, res) => {
       
       
        if (err) {
            console.log("热点新闻抓取失败-${err}");
        } else {
            console.log("爬取新闻成功!");
            if (myURL == "https://vedio.sina.com/") return;
            getHotNews(res, myURL);

        }
        
    })
}


function getHotNews(res, myURL) {
    let $ = myCheerio.load(res.text, { decodeEntities: true });
    
    let fetch = {};
    fetch.title = "";
    fetch.content = "";
    fetch.keywords = "";
    fetch.publish_date = "";
    fetch.author = "";
    fetch.desc = "";
    fetch.source_name = source_name;
    fetch.source_encoding = myEncoding;
    fetch.crawltime = new Date;
    fetch.url = myURL;

    fetch.title = $('meta[property=\"og:title\"]').eq(0).attr("content");
    if (fetch.title == "") fetch.title = source_name;

    fetch.content = $('.article').text();

    fetch.keywords = $('meta[name=\"keywords\"]').eq(0).attr("content");

    fetch.publish_date = $('meta[name=\"weibo: article:create_at\"]').eq(0).text().replace(/\s*/g,"").substr(0, 10);

    fetch.author = $('meta[property=\"article:author\"]').eq(0).attr("content");

    fetch.desc = $('meta[property=\"og:description\"]').eq(0).attr("content");

    let filename = source_name + "_" + (new Date()).toFormat("YYYY-MM-DD") +
    "_" + myURL.slice(myURL.lastIndexOf('/') + 1, myURL.lastIndexOf('.')) +".json";
   
    let fetchAddSql = 'INSERT INTO fetches(url,source_name,source_encoding,title,' +
    'keywords,author,publish_date,crawltime,content) VALUES(?,?,?,?,?,?,?,?,?)';
let fetchAddSql_Params = [fetch.url, fetch.source_name, fetch.source_encoding,
    fetch.title, fetch.keywords, fetch.author, fetch.publish_date,
    fetch.crawltime.toFormat("YYYY-MM-DD HH24:MI:SS"), fetch.content
];

 
mysql.query(fetchAddSql, fetchAddSql_Params, function(qerr, vals, fields) {
    if (qerr) {
          console.log(qerr);
    }
        console.log("插入数据库成功!");
});
}; 
